/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2012-2-8
 */
package com.autotag.data.crawler;

import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.util.Iterator;
import java.util.List;

import com.autotag.dao.AbstractProviderDao;
import com.autotag.entity.AbstractProvider;

import fanglu.util.FileDealer;
import fanglu.util.SQLDealer;

public class ProviderDetailCrawler {
	private static SimpleHTMLFetcher simpleHTMLFetcher = new SimpleHTMLFetcher();
	public static boolean fetchProvider(AbstractProvider thisAbstractProvider, String baseDir, String encoding){
		boolean result = true;
		if (thisAbstractProvider.getStatus() == AbstractProvider.DETAIL_PROVIDER){
			return true;
		}
		String thisLowerName = thisAbstractProvider.getName().toLowerCase();
		String subDirName = "/other";
		if (thisLowerName.toLowerCase().charAt(0) >= 'a' && thisLowerName.toLowerCase().charAt(0) <= 'z'){
			subDirName = "/" + thisLowerName.toLowerCase().charAt(0);
		}
		String thisDirName = baseDir + subDirName + "/" + thisAbstractProvider.getName();
		File thisDir = new File(thisDirName);
		if (!thisDir.exists()){
			thisDir.mkdirs();
		}
		String urlString = "http://webservices.seekda.com/providers/" + thisAbstractProvider.getName() + "/";
		StringBuffer sb = new StringBuffer();
		int statusCode = simpleHTMLFetcher.simpleGetPageContent(urlString, sb, encoding);
		if (statusCode == 200 && sb.indexOf("Service Temporarily Unavailable") < 0){
			String providerFileName = thisDirName + "/provider.htm";
			try {
				FileDealer.saveFile(sb.toString(), providerFileName, encoding, false);
				if (thisAbstractProvider.getServiceNum() > 0){
					int thisPageNum = (thisAbstractProvider.getServiceNum() - 1) / 10 + 1;
					for (int i=0; i<thisPageNum; ++i){
						String subUrlString = urlString + "?tab=servicelist&p=" + (i*10);
						sb.delete(0, sb.length());						
						statusCode = simpleHTMLFetcher.simpleGetPageContent(subUrlString, sb, encoding);
						if (statusCode == 200 && sb.indexOf("Service Temporarily Unavailable") < 0){
							String subProviderFileName = thisDirName + "/list" + (i+1) + ".htm";
							FileDealer.saveFile(sb.toString(), subProviderFileName, encoding, false);
						}else{
							result = false;
							break;
						}
					}
				}
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				result = false;
			}
		}else{
			result = false;
		}
		return result;
	}
	
	public static void crawlAllProviderDetail(String baseDir, String encoding, String dbURL, String user, String pwd){
		Connection conn = SQLDealer.getDbConnection(dbURL, user, pwd);
		List<AbstractProvider> providerList = AbstractProviderDao.getAllAbstractProvider(conn);
		Iterator<AbstractProvider> itor = providerList.iterator();
		int totalSize = providerList.size();
		int thisIndex = 0;
		while (itor.hasNext()){
			AbstractProvider thisProvider = itor.next();
			int retryTime = 0;
			while (!fetchProvider(thisProvider, baseDir, encoding)){
				System.out.println((thisIndex+1) + " / " + totalSize + " is failed! retry!");
				try {
					Thread.sleep(60000);
				} catch (InterruptedException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				++ retryTime;
				if (retryTime > 10){
					break;
				}
			}
			if (retryTime > 10){
				System.out.println((thisIndex+1) + " / " + totalSize + " is droped!");
			}else{
				thisProvider.setStatus(AbstractProvider.DETAIL_PROVIDER);
				AbstractProviderDao.updateProvider(conn, thisProvider);
				System.out.println((thisIndex+1) + " / " + totalSize + " is OK!");
			}
			++ thisIndex;
		}
	}
	
	public static void main(String args[]){
		crawlAllProviderDetail("d:\\seekda_data\\provider_detail", "utf8",
				"jdbc:mysql://localhost:3306/autotag", "root", "ftc86524");
	}
}
